############################################################################################################################################################################################
### Setup 

# Source setup file, define which congresses
setwd('~/Dropbox/Projects/Dissertation/Bill Text and Agenda Control/JOP Replication Files')
source('text_setup.R')
congs <- 104:114

# Load formatted analysis data
load('Data/BTAC_predNonFigures.RData')

# Load bill actions data
baFull <- fread('Data/billActions_93to114.csv', data.table=FALSE, stringsAsFactors = FALSE) %>%
  filter(cong %in% congs)
ba <- baFull # For subsetting



############################################################################################################################################################################################
### Stage at which bills died

### Laws

# Which bills became law?
law <- filter(ba, str_detect(status, 'ENACTED')) %>%
  pull(id) %>%
  unique()

# Take those which became law out of ba, since we already know about those
ba <- filter(ba, !id %in% law)


### Passed at least one chamber

# Which bills passed in one chamber or the other
pass <- ba %>% 
  group_by(id) %>%
  dplyr::summarise(
    passH=unique(house_passage_result),
    passS=unique(senate_passage_result)
  ) %>%
  ungroup() %>%
  mutate(passH=car::recode(passH, "NA='fail'")) %>%
  mutate(passS=car::recode(passS, "NA='fail'")) %>%
  mutate(type=str_extract(id, '[a-z]+')) %>%
  mutate(orig_chamber=ifelse(str_detect(type, '^h'), 'house', 'senate')) %>%
  mutate(passBothNoLaw=ifelse(passH=='pass' & passS=='pass', 1, 0))
passList <- pass$id[pass$passH=='pass' | pass$passS=='pass']

# Take out passed bills from ba, since we already know those passed
ba %<>% filter(!id %in% passList)


### Had a vote

# Which bills failed a final passage vote?
failH <- filter(ba, status=='FAIL:ORIGINATING:HOUSE') %>% pull(id) %>% unique()
failS <- filter(ba, status=='FAIL:ORIGINATING:SENATE') %>% pull(id) %>% unique()  

# Which bills failed cloture?
failClot <- filter(ba, status=='PROV_KILL:CLOTUREFAILED') %>% pull(id) %>% unique()

# Which bills failed suspension of rules?
failSus <- filter(ba, status=='PROV_KILL:SUSPENSIONFAILED') %>% pull(id) %>% unique()

# Others
oth <- filter(ba, !is.na(result)) %>% pull(id) %>% unique

# Concatenate
fail <- c(failH, failS, failClot, failSus, oth) %>% unique() # One overlap (112_s_2343)

# Remove from ba
ba %<>% filter(!id %in% fail)


### Committee report

# Which bills were reported out of committee?
report <- c(filter(ba, status=='REPORTED') %>% pull(id) %>% unique(),
            filter(ba, str_detect(tolower(text), 'reported')) %>% pull(id) %>% unique()
)

# Take out reported bills from ba
ba %<>% filter(!id %in% report)


### Were hearings held?

# Which bills had hearings?
hear <- c(filter(ba, type=='hearings') %>% pull(id) %>% unique(),
          filter(ba, str_detect(tolower(text), 'hearings held')) %>% pull(id) %>% unique(),
          filter(ba, str_detect(tolower(text), 'held hearings')) %>% pull(id) %>% unique()
)

# Remove from ba
ba %<>% filter(!id %in% hear)


### Referreral (at this point we're only looking at those with a 'referred' status)

# Requested executive comment
comm <- filter(ba, str_detect(tolower(text), 'executive comment')) %>% pull(id) %>% unique()

# Also check for "forwarded by subcommittee" and "mark-up session held"
mark <- filter(ba, str_detect(tolower(text), 'mark-up session')) %>% pull(id) %>% unique()
forward <- filter(ba, str_detect(tolower(text), 'forwarded by')) %>% pull(id) %>% unique()

# Did any have a reading?
read <- filter(ba, str_detect(tolower(text), 'read ')) %>% pull(id) %>% unique()

# Bills referred to a subcommittee
subC <- filter(ba, str_detect(tolower(text), 'subcommittee') ) %>% pull(id) %>% unique()

# Also check 'rereferred', 'provision incorporated', 'provisions...incorporated', 'further action', 'see'
reref <- filter(ba, str_detect(tolower(text), 'rereferred')) %>% pull(id) %>% unique()
ba$text <- str_replace_all(ba$text, 'consideratiion', 'consideration')
prov <- filter(ba, str_detect(tolower(text), 'provision')) %>% 
  filter(!str_detect(tolower(text), 'consideration of such')) %>% pull(id) %>% unique()
furth <- c(filter(ba, str_detect(tolower(text), '^see ')) %>% pull(id) %>% unique(),
           filter(ba, str_detect(tolower(text), '^for further action ')) %>% pull(id) %>% unique()
)

# Merge all the pre-hearing categories together
ref <- c(comm, mark, forward, read, subC, reref, prov, furth) %>% unique()

# What about those that didn't have any of these? This is what's left
none <- filter(ba, !id %in% ref) %>% pull(id) %>% unique()
baNo <- filter(ba, id %in% none)



#############################################################################################################################################################################################
### Match designations to floor and pre-floor bills

# Reformat bill ID variable in pred and predV to match the
pred$billID <- with(pred, paste0(cong, '_', type, '_', str_extract(id, '[0-9]+')))
predV$billID <- with(predV, paste0(cong, '_', type, '_', str_extract(billnum, '[0-9]+')))


# Make indicators for all the prefloor bill categories
pred$reported <- ifelse(pred$billID %in% report, 1, 0)  # Bills that were reported out of committee (but no vote)
pred$hearing <- ifelse(pred$billID %in% hear, 1, 0)  # Bills that had a hearing
pred$execComm <- ifelse(pred$billID %in% comm, 1, 0)  # Bills receiving executive comment
pred$markup <- ifelse(pred$billID %in% mark, 1, 0)  # Bills with a markup session
pred$forwarded <- ifelse(pred$billID %in% forward, 1, 0)  # Bills forwarded to full committee by subcommittee
pred$reading <- ifelse(pred$billID %in% read, 1, 0)  # Bills which were read
pred$subc <- ifelse(pred$billID %in% subC, 1, 0)  # Bills referred to a subcommittee
pred$reref <- ifelse(pred$billID %in% reref, 1, 0)  # Bills rereferred to committee
pred$prov <- ifelse(pred$billID %in% prov, 1, 0)  # Bills where some provisions were enacted in a different bills
pred$furth <- ifelse(pred$billID %in% furth, 1, 0)  # Bills with a "for further action see Bill X" action
pred$oth <- ifelse(pred$furth==1 | pred$prov==1, 1, 0)  # Bills where either provisions were enacted in a different bill or we're directed toward another bill
pred$none <- ifelse(pred$billID %in% none, 1, 0)  # Blls with none of the other categorizations
pred$missing <- ifelse(pred$report==0 & pred$hearing==0 & pred$execComm==0 & pred$markup==0 & 
                         pred$forwarded==0 & pred$reading==0 & pred$subc==0 & pred$reref==0 & 
                         pred$oth==0 & pred$none==0, 1, 0) # Bills that fit none of the categories



#############################################################################################################################################################################################
### Figure 6

### Set up data

# Get proportions
stage <- pred %>%
  filter(missing==0 & hitchhiker==0 & oth==0 &  majSup==0) %>%
  group_by(cong, maj_int, chamber) %>%
  dplyr::summarise(
    obs=n(),
    reported=sum(reported)/obs,
    hearing=sum(hearing)/obs,
    execComm=sum(execComm)/obs,
    markup=sum(markup)/obs,
    forwarded=sum(forwarded)/obs,
    reading=sum(reading)/obs,
    subc=sum(subc)/obs,
    reref=sum(reref)/obs,
    oth=sum(none)/obs
  ) %>%
  ungroup()

# Split by chamber, remove categorizations that don't happen in each chamber, make a catchall for "further committee action"
stageHmin <- filter(stage, chamber=='house' & maj_int==0) %>%
  dplyr::select(-c(reading, reref)) %>%
  mutate(furtherAction=hearing+execComm+markup+forwarded)
stageHmaj <- filter(stage, chamber=='house' & maj_int==1) %>%
  dplyr::select(-c(reading, reref)) %>%
  mutate(furtherAction=hearing+execComm+markup+forwarded)
stageSmin <- filter(stage, chamber=='senate' & maj_int==0) %>%
  dplyr::select(-c(oth, reref, markup, forwarded)) %>%
  mutate(furtherAction=hearing+execComm+subc)
stageSmaj <- filter(stage, chamber=='senate' & maj_int==1) %>%
  dplyr::select(-c(oth, reref, markup, forwarded)) %>%
  mutate(furtherAction=hearing+execComm+subc)


### Plot and save

# Turn on graphics device, set up mfrow
png(filename='Figures/fg6.png', width=10, height=10, res=1200, units='in')
par(mfrow=c(2,2))

# Plot House categories, minority-introduced
cols <- brewer.pal(5, 'Greys')
plot(sort(unique(stageHmin$cong)), stageHmin$reported, pch=16, type='n', ylim=c(0,1), yaxt='n',
     ylab='Proportion of Pre-Floor Bills', xlab='Congress', xaxt='n', main='House, Minority-Introduced')
grid(nx=NA, ny=NULL)
lines(sort(unique(stageHmin$cong)), stageHmin$reported, pch=16, type='b', col=cols[2], lwd=2)
lines(sort(unique(stageHmin$cong)), stageHmin$furtherAction, pch=16, type='b', col=cols[3], lwd=2)
lines(sort(unique(stageHmin$cong)), stageHmin$subc, pch=16, type='b', col=cols[4], lwd=2)
lines(sort(unique(stageHmin$cong)), stageHmin$oth, pch=16, type='b', col=cols[5], lwd=2)
legend(x='topright', legend=c('Reported', 'Committee Action', 'Referred to Subcommittee', 'No Action'), 
       col=cols[2:5], lty=rep(1, 4), pch=rep(16, 4), bty='n')
axis(side=1, at=sort(unique(stageHmin$cong)), las=2)
axis(side=2, at=seq(0, 1, 0.2), las=2)

# Plot House categories, majority-introduced
plot(sort(unique(stageHmaj$cong)), stageHmaj$reported, pch=16, type='n', ylim=c(0,1), yaxt='n',
     ylab='Proportion of Pre-Floor Bills', xlab='Congress', xaxt='n', main='House, Majority-Introduced')
grid(nx=NA, ny=NULL)
lines(sort(unique(stageHmaj$cong)), stageHmaj$reported, pch=16, type='b', col=cols[2], lwd=2)
lines(sort(unique(stageHmaj$cong)), stageHmaj$furtherAction, pch=16, type='b', col=cols[3], lwd=2)
lines(sort(unique(stageHmaj$cong)), stageHmaj$subc, pch=16, type='b', col=cols[4], lwd=2)
lines(sort(unique(stageHmaj$cong)), stageHmaj$oth, pch=16, type='b', col=cols[5], lwd=2)
legend(x='topright', legend=c('Reported', 'Committee Action', 'Referred to Subcommittee', 'No Action'), 
       col=cols[2:5], lty=rep(1, 4), pch=rep(16, 4), bty='n')
axis(side=1, at=sort(unique(stageHmaj$cong)), las=2)
axis(side=2, at=seq(0, 1, 0.2), las=2)

# Plot Senate categories, minority-introduced
plot(sort(unique(stageSmin$cong)), stageSmin$reported, pch=16, type='n', ylim=c(0,1), yaxt='n',
     ylab='Proportion of Pre-Floor Bills', xlab='Congress', xaxt='n', main='Senate, Minority-Introduced')
grid(nx=NA, ny=NULL)
lines(sort(unique(stageSmin$cong)), stageSmin$reported, pch=16, type='b', col=cols[2], lwd=2)
lines(sort(unique(stageSmin$cong)), stageSmin$furtherAction, pch=16, type='b', col=cols[3], lwd=2)
lines(sort(unique(stageSmin$cong)), stageSmin$reading, pch=16, type='b', col=cols[5], lwd=2)
legend(x='right', legend=c('Reported', 'Committee Action', 'No Action'), 
       col=cols[c(2, 3, 5)], lty=rep(1, 4), pch=rep(16, 4), bty='n')
axis(side=1, at=sort(unique(stageSmin$cong)), las=2)
axis(side=2, at=seq(0, 1, 0.2), las=2)

# Plot Senate categories, majority-introduced
plot(sort(unique(stageSmaj$cong)), stageSmaj$reported, pch=16, type='n', ylim=c(0,1), yaxt='n',
     ylab='Proportion of Pre-Floor Bills', xlab='Congress', xaxt='n', main='Senate, Majority-Introduced')
grid(nx=NA, ny=NULL)
lines(sort(unique(stageSmaj$cong)), stageSmaj$reported, pch=16, type='b', col=cols[2], lwd=2)
lines(sort(unique(stageSmaj$cong)), stageSmaj$furtherAction, pch=16, type='b', col=cols[3], lwd=2)
lines(sort(unique(stageSmaj$cong)), stageSmaj$reading, pch=16, type='b', col=cols[5], lwd=2)
legend(x='right', legend=c('Reported', 'Committee Action', 'No Action'), 
       col=cols[c(2, 3, 5)], lty=rep(1, 4), pch=rep(16, 4), bty='n')
axis(side=1, at=sort(unique(stageSmaj$cong)), las=2)
axis(side=2, at=seq(0, 1, 0.2), las=2)

# Turn off graphics device, reset par
dev.off()
par(mfrow=c(1,1))
